{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4o7b4VTDswyu"
   },
   "source": [
    "# mnist\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 565
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6942,
     "status": "ok",
     "timestamp": 1539652110962,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "cMp2gQd2pv5_",
    "outputId": "b1b8b753-515b-4eea-af2a-cc3025f91102"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-7c0ffcdda0da>:12: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use urllib or similar directly.\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/input_data/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/tensorflow/input_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting /tmp/tensorflow/input_data/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting /tmp/tensorflow/input_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: __init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "FLAGS = None\n",
    "\n",
    "mnist = input_data.read_data_sets('/tmp/tensorflow/input_data', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4752,
     "status": "ok",
     "timestamp": 1539652119874,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "nj3iY1-7rNKi",
    "outputId": "5991d87d-47ee-42e3-e792-2e4fa710c279"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t10k-images-idx3-ubyte.gz  train-images-idx3-ubyte.gz\n",
      "t10k-labels-idx1-ubyte.gz  train-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "!ls /tmp/tensorflow/input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7443,
     "status": "ok",
     "timestamp": 1539609653389,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "1uZJ1pZaszZg",
    "outputId": "a22bbc85-4716-4f19-8587-34b25bc6d705"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9227\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W = tf.Variable(tf.zeros([784, 10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "y = tf.matmul(x, W) + b\n",
    "\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LXEKnReiwi_9"
   },
   "source": [
    "# 增加隐藏层"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lOdJAxKv4GJJ"
   },
   "source": [
    "### 增加一层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8948,
     "status": "ok",
     "timestamp": 1539660154693,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "-MfGHqNctVcG",
    "outputId": "4142c41e-e300-4587-bc10-68cb0a4a5da3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9777\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rZTM8w166AH9"
   },
   "source": [
    "### 在增加一层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10087,
     "status": "ok",
     "timestamp": 1539660183447,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "F94sSjWFtVLS",
    "outputId": "e73b3f0e-3c00-4484-d9e6-23af8d8fa405"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9783\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 300], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1)\n",
    "L2 = tf.nn.relu(tf.matmul(L1, W2) + b2)\n",
    "\n",
    "W3 = tf.Variable(tf.truncated_normal([300, 10], stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L2, W3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zjOkI9pz6NUw"
   },
   "source": [
    "### 在在增加一层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10326,
     "status": "ok",
     "timestamp": 1539660201575,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "ahO0CYdb24eT",
    "outputId": "396bd6fe-1177-4c8a-bdb2-4294c0e04c19"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9721\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 300], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1)\n",
    "L2 = tf.nn.relu(tf.matmul(L1, W2) + b2)\n",
    "\n",
    "W3 = tf.Variable(tf.truncated_normal([300, 100], stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([100]) + 0.1)\n",
    "L3 = tf.nn.relu(tf.matmul(L2, W3) + b3)\n",
    "\n",
    "W4 = tf.Variable(tf.truncated_normal([100, 10], stddev=0.1))\n",
    "b4 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L3, W4) + b4\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rhZ0MCeT5dAr"
   },
   "source": [
    "增加隐藏层确实能够增加准确率，但是增加的越多，后面增加的不明显了。这里我们增加一层就够了"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nlkUnRsWw-bD"
   },
   "source": [
    "#  激活函数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FSh5wj1T_dSF"
   },
   "source": [
    "### tanh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10113,
     "status": "ok",
     "timestamp": 1539660247844,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "3aHT__cV4Cpj",
    "outputId": "99cb9c71-051c-441d-beaf-be87532598dc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9762\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.tanh(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZvBwmkiA_lnw"
   },
   "source": [
    "### sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9612,
     "status": "ok",
     "timestamp": 1539660315591,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "NYo9tTsFtVUx",
    "outputId": "9eb587bd-0ebc-4c4f-bce1-db6e86be91c9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9547\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.sigmoid(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Tb6MhNTrA0o8"
   },
   "source": [
    "### leaky_relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 23652,
     "status": "ok",
     "timestamp": 1539660305948,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "E33iv_InA3fT",
    "outputId": "31a7c6c3-6923-4bb3-b880-3c6870127adf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9764\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.leaky_relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TdO9Vr8FCupA"
   },
   "source": [
    "### PReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10608,
     "status": "ok",
     "timestamp": 1539660430541,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "XTtnHR-NG1S8",
    "outputId": "e5014c17-f265-4195-8ad6-bf10403cd7d3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9806\n"
     ]
    }
   ],
   "source": [
    "def prelu(_x, scope=None):\n",
    "    with tf.variable_scope(name_or_scope=scope, default_name=\"prelu\"):\n",
    "        _alpha = tf.get_variable(\"prelu\", \n",
    "                                 shape=_x.get_shape()[-1],\n",
    "                                 dtype=_x.dtype, \n",
    "                                 initializer=tf.constant_initializer(0.1))\n",
    "        return tf.maximum(0.0, _x) + _alpha * tf.minimum(0.0, _x)\n",
    "\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = prelu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "250TkZZm6dDQ"
   },
   "source": [
    "准确率超过98%"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LPSjC7FhEuax"
   },
   "source": [
    "### elu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9363,
     "status": "ok",
     "timestamp": 1539660528302,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "KHNDULo0EvdE",
    "outputId": "be7ed5a3-2699-4908-9055-cb8fdbf300c9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9736\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.elu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1pVzryT_C1P6"
   },
   "source": [
    "### selu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8942,
     "status": "ok",
     "timestamp": 1539660550944,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "s22PPhbgC2Wv",
    "outputId": "8b50dbfa-afb7-4dad-b9c2-df5d864593ee"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9603\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.selu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h1C9x_lzFS4V"
   },
   "source": [
    "### swish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9128,
     "status": "ok",
     "timestamp": 1539660571649,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "ozO5A-I5FT-8",
    "outputId": "b802e8a0-df9e-4b64-818e-676108749f34"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9801\n"
     ]
    }
   ],
   "source": [
    "def swish(x, b = 1):\n",
    "    return x * tf.nn.sigmoid(b * x)\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = swish(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fS8cs50L6_Nb"
   },
   "source": [
    "准确率也达到了98%以上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2-0VhPdWBDg5"
   },
   "source": [
    "### relu6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8980,
     "status": "ok",
     "timestamp": 1539660610922,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "rPA0ktB3Bl7T",
    "outputId": "6ee4a14d-fa91-4546-eef9-1b2b2b689a92"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9795\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu6(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wq5AmKzQDabH"
   },
   "source": [
    "以上激活函数准备率都不错，prelu、swish已经达到了98%以上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "K8CxyGIwxPP0"
   },
   "source": [
    "# 正则化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TbSu3gNkK7-s"
   },
   "source": [
    "### L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9829,
     "status": "ok",
     "timestamp": 1539661103875,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "r5l_UG_dxTln",
    "outputId": "5cf79952-3d05-4d05-9cef-e10d8b30466f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9358\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "# L1正则化方法\n",
    "regularizer = tf.contrib.layers.l1_regularizer(0.001)\n",
    "regularization = regularizer(W1) + regularizer(W2)  \n",
    "\n",
    "cross_entroy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "loss = cross_entroy + regularization\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uFviPNDDK-QS"
   },
   "source": [
    "### L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9360,
     "status": "ok",
     "timestamp": 1539661181133,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "2H-bgvoILAbY",
    "outputId": "7d72c25a-a8ad-4a43-d6da-16d12fe3bda7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9733\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "# L2正则化方法\n",
    "regularizer = tf.contrib.layers.l2_regularizer(0.001)\n",
    "regularization = regularizer(W1) + regularizer(W2)\n",
    "\n",
    "cross_entroy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "loss = cross_entroy + regularization\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Egyqb_QixUDi"
   },
   "source": [
    "# 初始化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oM63lptnltEV"
   },
   "source": [
    "### tf.random_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9187,
     "status": "ok",
     "timestamp": 1539661553229,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "tTee0l18kUm2",
    "outputId": "57225cac-91d0-4ec5-e1d8-f3a94c730c0d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9778\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.random_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.random_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9449,
     "status": "ok",
     "timestamp": 1539665239697,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "5mI-JAbeMEZb",
    "outputId": "d944654f-6650-4b14-e993-b9e91134c4e1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9366\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.random_normal([784, 500], stddev=1.0))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.random_normal([500, 10], stddev=1.0))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9MIv3ctvMNUp"
   },
   "source": [
    "### tf.truncated_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9264,
     "status": "ok",
     "timestamp": 1539665254553,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "Lknr-hGB-NPz",
    "outputId": "73ad52c4-61a3-4131-d49a-d025fd2f788d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9296\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=1.0))\n",
    "b1 = tf.Variable(tf.constant(0.1, shape=[500]))\n",
    "# b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=1.0))\n",
    "b1 = tf.Variable(tf.constant(0.1, shape=[10]))\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "z3D1RaDBpzRS"
   },
   "source": [
    "### bias初始化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fsnJnoVNp_PQ"
   },
   "source": [
    "random_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9993,
     "status": "ok",
     "timestamp": 1539672984971,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "KRxYir-pp3z1",
    "outputId": "e34072d2-57af-47f6-e572-ab9a1752a5a7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9771\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.random_normal([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.random_normal([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ss5eENSM6qp9"
   },
   "source": [
    "# 探索如下超参数设置"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "K2GAEqZw6uJh"
   },
   "source": [
    "## 隐层数量\n",
    "\n",
    "同上\"增加隐藏层\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sdeYjMLujlvD"
   },
   "source": [
    "## 各隐层中神经元数量"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Yz9eO5e7HZGe"
   },
   "source": [
    "### 隐藏层只有一层"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WCHLulT0IJs1"
   },
   "source": [
    "隐藏层神经元数量500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 22220,
     "status": "ok",
     "timestamp": 1539660044019,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "8nwutxHUpVSM",
    "outputId": "fa976cbd-7b97-42ee-f43a-6b1a4a6968b7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.979\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(64)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tgq4vmfTILYy"
   },
   "source": [
    "隐藏层神经元数量100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8412,
     "status": "ok",
     "timestamp": 1539663999408,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "fEvs1cppH-Zn",
    "outputId": "e9b64981-6922-41dd-a30b-fef68a2a27bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.974\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 100], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([100]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([100, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(64)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5PRr5lVnI1oH"
   },
   "source": [
    "这里，隐藏层神经元数量越多准确率越高"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Aj48VxW6H0R0"
   },
   "source": [
    "### 隐藏层两层"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ie8aKa99ISMR"
   },
   "source": [
    "隐藏层第一层神经元数量500，第二层300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10736,
     "status": "ok",
     "timestamp": 1539664163160,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "319pr8veH0qE",
    "outputId": "0041d82c-b635-43b1-e505-5729be0174fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9788\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 300], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1)\n",
    "L2 = tf.nn.relu(tf.matmul(L1, W2) + b2)\n",
    "\n",
    "W3 = tf.Variable(tf.truncated_normal([300, 10], stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L2, W3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xwVbUCXBKyky"
   },
   "source": [
    "隐藏层第一层神经元数量500，第二层100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10241,
     "status": "ok",
     "timestamp": 1539664187930,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "LgxQ9rusIsC8",
    "outputId": "4cae80ab-24aa-42b6-86d2-ac2667eb2ad0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9812\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 100], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([100]) + 0.1)\n",
    "L2 = tf.nn.relu(tf.matmul(L1, W2) + b2)\n",
    "\n",
    "W3 = tf.Variable(tf.truncated_normal([100, 10], stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L2, W3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sNdB_WTfjl8y"
   },
   "source": [
    "## 学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9255,
     "status": "ok",
     "timestamp": 1539661711185,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "F8pABm7RjvId",
    "outputId": "e10b5c58-5e26-4c77-a615-38a1fbc34cea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9783\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])    \n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(1).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9131,
     "status": "ok",
     "timestamp": 1539661726623,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "mKbRRGz8yE34",
    "outputId": "652beb7a-5e32-4198-a8d3-dc4b8ccc09c8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9652\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])   \n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.1).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Oy7B9irzzoH2"
   },
   "source": [
    "### 自适应学习速率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9681,
     "status": "ok",
     "timestamp": 1539663526637,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "gHcMmSryGGP1",
    "outputId": "b9af80e7-aca4-495e-a267-f50257025b41"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9784\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "cross_entroy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "starter_learning_rate = 1.0  #初始学习速率\n",
    "mnist.train.num_examples / 100 ＝ 550\n",
    "learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, mnist.train.num_examples / 100, 0.96, staircase=False)\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9533,
     "status": "ok",
     "timestamp": 1539661742031,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "lU7zGZjFyVhr",
    "outputId": "6f06c6c7-9220-47d3-9a61-45a15b5fc219"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9809\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "cross_entroy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "starter_learning_rate = 1.0  #初始学习速率\n",
    "learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 100, 0.96, staircase=False)\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xh6qEcX9_lzf"
   },
   "source": [
    "改成自适应学习率后，准确率能超过98%"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "exSUVvEkjmIs"
   },
   "source": [
    "## 正则化因子\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RW9qLCL_KO8Z"
   },
   "source": [
    "同上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eBvji9-8jmTZ"
   },
   "source": [
    "## 权重初始化分布参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PGgpZIp6PAvN"
   },
   "source": [
    "### xavier_initializer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9560,
     "status": "ok",
     "timestamp": 1539672102991,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "tTg5ESISj4h7",
    "outputId": "b64f1e50-03c9-48dc-e6b9-341bad81b8a8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.979\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.get_variable('w11_xaiver', [784, 500], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.get_variable('w22_xaiver', [500, 10], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "i72byx2po3aI"
   },
   "source": [
    "在增加一层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11170,
     "status": "ok",
     "timestamp": 1539672520098,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "Txy0x5wTne_T",
    "outputId": "061eceac-7103-4eca-cdf2-7795c035d4e0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9795\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.get_variable('w41_xaiver', [784, 500], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.get_variable('w42_xaiver', [500, 300], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1)\n",
    "L2 = tf.nn.relu(tf.matmul(L1, W2) + b2)\n",
    "\n",
    "W3 = tf.get_variable('w43_xaiver', [300, 10], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L2, W3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XXrCCe8nnFVC"
   },
   "source": [
    "### variance_scaling_initializer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9758,
     "status": "ok",
     "timestamp": 1539672600925,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "FBtypzvdnMtF",
    "outputId": "96ee60d0-c573-44fe-eb80-8bf548bbbab3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.98\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.get_variable('w1_scale', [784, 500], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.get_variable('w2_scale', [500, 10], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hNTKozMvo941"
   },
   "source": [
    "在增加一层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10861,
     "status": "ok",
     "timestamp": 1539672687293,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "iolmx9fUo2Wn",
    "outputId": "d1df1b0a-f6ee-42ff-fbb2-d5e567265715"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9801\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.get_variable('w31_scale', [784, 500], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.get_variable('w32_scale', [500, 300], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1)\n",
    "L2 = tf.nn.relu(tf.matmul(L1, W2) + b2)\n",
    "\n",
    "W3 = tf.get_variable('w33_scale', [300, 10], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L2, W3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(3000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7AKFt1gcAU8k"
   },
   "source": [
    "## 增加迭代次数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 67966,
     "status": "ok",
     "timestamp": 1539661938448,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "_h_N77wC_5Y2",
    "outputId": "00d3bb56-f9eb-4622-94e2-1bcf10790a5a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9822\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(tf.truncated_normal([784, 500], stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([500, 10], stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(30000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6OMyf3r0AaBj"
   },
   "source": [
    "增加迭代次数准备率也能达到98%以上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jawCM528uRNB"
   },
   "source": [
    "# 最后"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 73134,
     "status": "ok",
     "timestamp": 1539674550054,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "17Cs_8b_tUFk",
    "outputId": "06631a6b-1422-4a30-adac-201db98efb2a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.984\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.get_variable('w11_scale', [784, 500], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.get_variable('w12_scale', [500, 10], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b2 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "y = tf.matmul(L1, W2) + b2\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "starter_learning_rate = 1.0  #初始学习速率\n",
    "learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 100, 0.96, staircase=False)\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(30000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 79742,
     "status": "ok",
     "timestamp": 1539675411322,
     "user": {
      "displayName": "sai chen",
      "photoUrl": "",
      "userId": "16203653011462049496"
     },
     "user_tz": -480
    },
    "id": "yUUuxD4_y6z7",
    "outputId": "42af9d77-caab-4c56-ca8c-b185c3caa8ef"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9838\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.get_variable('w61_scale', [784, 500], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b1 = tf.Variable(tf.zeros([500]) + 0.1)\n",
    "L1 = tf.nn.relu(tf.matmul(x, W1) + b1)\n",
    "\n",
    "W2 = tf.get_variable('w62_scale', [500, 300], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b2 = tf.Variable(tf.zeros([300]) + 0.1)\n",
    "L2 = tf.nn.relu(tf.matmul(L1, W2) + b2)\n",
    "\n",
    "W3 = tf.get_variable('w63_scale', [300, 10], initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "b3 = tf.Variable(tf.zeros([10]) + 0.1)\n",
    "\n",
    "y = tf.matmul(L2, W3) + b3\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "cross_entroy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "starter_learning_rate = 1.0  #初始学习速率\n",
    "learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 100, 0.96, staircase=False)\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entroy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "for _ in range(30000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    \n",
    "    \n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images, \n",
    "                                    y_: mnist.test.labels}))\n",
    "\n",
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "4o7b4VTDswyu",
    "zjOkI9pz6NUw",
    "FSh5wj1T_dSF",
    "ZvBwmkiA_lnw",
    "Tb6MhNTrA0o8",
    "TdO9Vr8FCupA",
    "LPSjC7FhEuax",
    "1pVzryT_C1P6",
    "2-0VhPdWBDg5",
    "K8CxyGIwxPP0",
    "TbSu3gNkK7-s",
    "uFviPNDDK-QS",
    "Egyqb_QixUDi",
    "oM63lptnltEV",
    "9MIv3ctvMNUp",
    "z3D1RaDBpzRS",
    "sdeYjMLujlvD",
    "Yz9eO5e7HZGe",
    "Aj48VxW6H0R0",
    "sNdB_WTfjl8y",
    "Oy7B9irzzoH2",
    "exSUVvEkjmIs",
    "eBvji9-8jmTZ",
    "PGgpZIp6PAvN",
    "XXrCCe8nnFVC",
    "7AKFt1gcAU8k"
   ],
   "name": "Home1.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
